{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    " \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.externals import joblib \n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import RidgeCV\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from lightgbm import LGBMRegressor\n",
    "import math\n",
    "from catboost import Pool, CatBoostRegressor\n",
    "%matplotlib inline\n",
    "\n",
    "# pd.set_option('display.max_colwidth',1000)\n",
    "# pd.set_option('display.height',1000)\n",
    "# pd.set_option('display.max_rows',500)\n",
    "# pd.set_option('display.max_columns',500)\n",
    "# pd.set_option('display.width',1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据预处理\n",
    "def clean_data(train_data_X):    \n",
    "    for j in range(len(np.array(train_data_X.columns))):\n",
    "        if train_data_X.columns[j] != \"ID\":\n",
    "            print(\"正在处理第%d列数据\"%j)\n",
    "            array = train_data_X[train_data_X.columns[j]]\n",
    "            num = 0\n",
    "            len_ = len(train_data_X[train_data_X.columns[j]])\n",
    "            for i in range(len_):   \n",
    "                std_ = array.std()\n",
    "                min_ = array.min()\n",
    "                max_ = array.max()\n",
    "                mean_ = array.mean()\n",
    "                num = array[i]\n",
    "                std_max =np.float(mean_+3*std_)\n",
    "                std_min = np.float(mean_-3*std_)\n",
    "                if (array[0]>std_max) | (array[0]<std_min):\n",
    "    #                 mean_del = mean_-(array[0]-mean_)/len_\n",
    "    #                 array[0] = mean_del\n",
    "                    array[0] = mean_\n",
    "                if (array[len_-1]>std_max) | (array[len_-1]<std_min):\n",
    "                    array[len_-1] = mean_\n",
    "                if i>0 and i<(len_-1):\n",
    "                    if (array[i]>std_max) | (array[i]<std_min):\n",
    "                        if (array[i-1]<std_max) and (array[i-1]>std_min) and (array[i+1]<std_max) and (array[i+1]>std_min):\n",
    "                            array[i]= np.float(array[i-1]+array[i+1])/2\n",
    "                        elif ((array[i-1]<std_max) and (array[i-1]>std_min))| (array[i+1]>std_max) | (array[i+1]<std_min): \n",
    "                            array[i] = array[i-1]\n",
    "                        elif (array[i-1]<std_max) | (array[i-1]>std_min)|((array[i+1]<std_max) and (array[i+1]>std_min)):\n",
    "                            array[i] = array[i+1]\n",
    "            train_data_X[train_data_X.columns[j]] = array\n",
    "        else:\n",
    "            pass\n",
    "            \n",
    "    return train_data_X\n",
    "def drop_all_outlier(df):\n",
    "    df.drop_duplicates(df.columns.drop('ID'), keep='first', inplace=True)\n",
    "    df.drop(df[(df.电压A > 800) | (df.电压A < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.电压B > 800) | (df.电压B < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.电压C > 800) | (df.电压C < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.现场温度 > 30) | (df.现场温度 < -30)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率A > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率B > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率C > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.风向 > 360)].index,inplace=True)\n",
    "    df.drop(df[(df.风速 > 20)].index,inplace=True)\n",
    "    return df\n",
    "# 生成数据\n",
    "def generate_train_data(train_data, test_data, poly=False, select=False):\n",
    "    y = train_data['发电量']\n",
    "    X = train_data.drop(['发电量','ID'], axis=1)\n",
    "    sub_data = test_data.drop(['ID'], axis=1)\n",
    "    \n",
    "    polynm = None\n",
    "    if poly:\n",
    "        from sklearn.preprocessing import PolynomialFeatures\n",
    "        polynm = PolynomialFeatures(degree=2, interaction_only=True)\n",
    "        X = polynm.fit_transform(X)\n",
    "        sub_data = polynm.transform(sub_data)\n",
    "        \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "    \n",
    "    sm = None\n",
    "    if select:\n",
    "        from sklearn.feature_selection import SelectFromModel\n",
    "        sm = SelectFromModel(GradientBoostingRegressor(random_state=2))\n",
    "        X_train = sm.fit_transform(X_train, y_train)\n",
    "        X_test = sm.transform(X_test)\n",
    "        sub_data = sm.transform(sub_data)\n",
    "        \n",
    "    return X_train, X_test, y_train, y_test, sub_data, sm, polynm\n",
    "\n",
    "def cal_score(mse):\n",
    "    if isinstance(mse, float):\n",
    "        return 1 / (1 + math.sqrt(mse))\n",
    "    else:\n",
    "        return np.divide(1, 1 + np.sqrt(mse))\n",
    "#  定义交叉验证函数  \n",
    "def cross_validation_test(models, train_X_data, train_y_data, cv=5):\n",
    "    model_name, mse_avg, score_avg = [], [], []\n",
    "    for i, model in enumerate(models):\n",
    "        print(i + 1,'- Model:', str(model).split('(')[0])\n",
    "        model_name.append(str(i + 1) + '.' + str(model).split('(')[0])\n",
    "        nmse = cross_val_score(model, train_X_data[i], train_y_data[i], cv=cv, scoring='neg_mean_squared_error')\n",
    "        avg_mse = np.average(-nmse)\n",
    "        scores = cal_score(-nmse)\n",
    "        avg_score = np.average(scores)\n",
    "        mse_avg.append(avg_mse)\n",
    "        score_avg.append(avg_score)\n",
    "        print('MSE:', -nmse)\n",
    "        print('Score:', scores)\n",
    "        print('Average XGB - MSE:', avg_mse, ' - Score:', avg_score, '\\n')\n",
    "    res = pd.DataFrame()\n",
    "    res['Model'] = model_name\n",
    "    res['Avg MSE'] = mse_avg\n",
    "    res['Avg Score'] = score_avg\n",
    "    return res\n",
    "\n",
    "# def add_newid(df):\n",
    "#     ID = df[\"ID\"]\n",
    "#     df[\"new_id\"]=(np.mod(ID,205))\n",
    "#     return df\n",
    "def add_avg(df):\n",
    "    array = np.array(df[\"平均功率\"])\n",
    "    newarray=[]\n",
    "    num = 0\n",
    "    for i in np.arange(len(array)):\n",
    "        for j in np.arange(10):\n",
    "            sum_1=0\n",
    "            for m in np.arange(j):\n",
    "                sum_1 += array[m]\n",
    "            if i<10:\n",
    "                num = sum_1/(j+1)\n",
    "            else:\n",
    "                num = (array[i-1]+array[i-2]+array[i-3]+array[i-4]+array[i-5]+array[i-6]+array[i-7]+array[i-8]+array[i-9])/9\n",
    "        newarray.append(num)\n",
    "    df[\"old平均功率\"] = newarray\n",
    "    return df\n",
    "def add_max(df):\n",
    "    array = np.array(df[\"平均功率\"])\n",
    "    newarray=[]\n",
    "    num = 0\n",
    "    for i in np.arange(len(array)):\n",
    "        for j in np.arange(10):\n",
    "            if i<3:\n",
    "                num =array[j]\n",
    "            elif i<10:\n",
    "                num = max(array[j-1],array[j-2],array[j-3])\n",
    "            else:\n",
    "                num = max(array[i-1],array[i-2],array[i-3],array[i-4],array[i-5],array[i-6],array[i-7],array[i-8],array[i-9])\n",
    "        newarray.append(num)\n",
    "    df[\"max_平均功率\"] = newarray\n",
    "    return df\n",
    "def add_min(df):\n",
    "    array = np.array(df[\"平均功率\"])\n",
    "    newarray=[]\n",
    "    num = 0\n",
    "    for i in np.arange(len(array)):\n",
    "        for j in np.arange(10):\n",
    "            if i<3:\n",
    "                num =array[j]\n",
    "            elif i<10:\n",
    "                num = max(array[j-1],array[j-2],array[j-3],)\n",
    "            else:\n",
    "                num = max(array[i-1],array[i-2],array[i-3],array[i-4],array[i-5],array[i-6],array[i-7],array[i-8],array[i-9])\n",
    "        newarray.append(num)\n",
    "    df[\"min_平均功率\"] = newarray\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(1,2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('../data/public.train.csv')\n",
    "test_data = pd.read_csv('../data/public.test.csv')\n",
    "# train_data = clean_data(train_data)\n",
    "\n",
    "# test_data = clean_data(test_data)\n",
    "# train_data = add_newid(train_data)\n",
    "# test_data = add_newid(test_data)\n",
    "# train_data = add_avg(train_data)\n",
    "# test_data = add_avg(test_data)\n",
    "df_result = pd.DataFrame()\n",
    "df_result['ID'] = list(test_data['ID'])\n",
    "special_missing_ID = test_data[test_data[(test_data == 0) | (test_data == 0.)].count(axis=1) > 13]['ID']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 异常值处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:16: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "cleaned_train_data = train_data.copy()\n",
    "cleaned_train_data = drop_all_outlier(cleaned_train_data)\n",
    "\n",
    "cleaned_sub_data = test_data.copy()\n",
    "cleaned_sub_data = drop_all_outlier(cleaned_sub_data)\n",
    "cleaned_sub_data_ID = cleaned_sub_data['ID']\n",
    "\n",
    "# all_data = pd.concat([train_data, test_data], axis=0).sort_values(by='ID').reset_index().drop(['index'], axis=1)\n",
    "# bad_feature = ['ID', '功率A', '功率B', '功率C', '平均功率', '现场温度', '电压A', '电压B', '电压C', '电流B', '电流C', '转换效率', '转换效率A', '转换效率B', '转换效率C']\n",
    "# bad_index = all_data[bad_feature][\n",
    "#     (all_data[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std()) | \n",
    "#     (all_data[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "# ].dropna(how='all').index\n",
    "\n",
    "\n",
    "all_data  = pd.concat([train_data, test_data], axis=0).sort_values(by='ID').reset_index().drop(['index'], axis=1)\n",
    "bad_feature = ['ID','功率A', '功率B', '功率C', '平均功率', '现场温度', '电压A', '电压B', '电压C', '电流B', '电流C', '转换效率', '转换效率A', '转换效率B', '转换效率C']\n",
    "bad_index1 = all_data[bad_feature][\n",
    "    (all_data[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std()) | \n",
    "    (all_data[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "].dropna(how='all').index\n",
    "bad_index2 = all_data[\n",
    "    ((all_data['电压A']<500)&(all_data['电压A']!=0))|\n",
    "    ((all_data['电压B']<500)&(all_data['电压B']!=0))|\n",
    "    ((all_data['电压C']<500)&(all_data['电压C']!=0))].index\n",
    "bad_index = pd.Int64Index(list(bad_index1)+list(bad_index2))\n",
    "# all_data.loc[np.concatenate([bad_index -1,bad_index,bad_index+1])].sort_values(by='ID', ascending=True)\n",
    "\n",
    "\n",
    "nn_bad_data = all_data.loc[np.concatenate([bad_index - 1, bad_index, bad_index + 1])].sort_values(by='ID', ascending=True).drop_duplicates()\n",
    "bad_data = all_data.loc[bad_index].sort_values(by='ID', ascending=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上下记录均值替代异常值\n",
    "for idx, line in bad_data.iterrows():\n",
    "    ID = line['ID']\n",
    "    col_index = line[bad_feature][ \n",
    "        (line[bad_feature] > all_data[bad_feature].mean() + 3 * all_data[bad_feature].std())| \n",
    "        (line[bad_feature] < all_data[bad_feature].mean() - 3 * all_data[bad_feature].std())\n",
    "    ].index\n",
    "    index = all_data[all_data['ID'] == ID].index\n",
    "    \n",
    "    before_offset = 1\n",
    "    while (idx + before_offset)in bad_index:\n",
    "        before_offset += 1\n",
    "\n",
    "    after_offset = 1\n",
    "    while (idx + after_offset) in bad_index:\n",
    "        after_offset += 1\n",
    "    \n",
    "    replace_value = (all_data.loc[index - before_offset, col_index].values + all_data.loc[index + after_offset, col_index].values) / 2\n",
    "    all_data.loc[index, col_index] = replace_value[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拆分数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "#拆分数据\n",
    "train_data = all_data.drop(all_data[all_data['ID'].isin(df_result['ID'])].index).reset_index().drop(['index'], axis=1)\n",
    "test_data = all_data[all_data['ID'].isin(df_result['ID'])].drop(['发电量'], axis=1).reset_index().drop(['index'], axis=1)\n",
    "len(train_data), len(test_data)\n",
    "# 去除重复值\n",
    "train_data = train_data.drop_duplicates(train_data.columns.drop('ID'), keep='first')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = add_avg(train_data)\n",
    "train_data = add_min(train_data)\n",
    "train_data = add_max(train_data)\n",
    "test_data = add_avg(test_data)\n",
    "test_data = add_max(test_data)\n",
    "test_data = add_min(test_data)\n",
    "\n",
    "cleaned_train_data = add_avg(cleaned_train_data)\n",
    "cleaned_sub_data = add_avg(cleaned_sub_data)\n",
    "cleaned_train_data = add_min(cleaned_train_data)\n",
    "cleaned_sub_data = add_min(cleaned_sub_data)\n",
    "cleaned_train_data = add_max(cleaned_train_data)\n",
    "cleaned_sub_data = add_max(cleaned_sub_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test, sub_data, sm, polynm = generate_train_data(train_data, test_data, poly=True, select=True)\n",
    "\n",
    "clean_X_train, clean_X_test, clean_y_train, clean_y_test, clean_sub_data, _, _ = generate_train_data(cleaned_train_data, cleaned_sub_data, poly=False, select=False)\n",
    "\n",
    "clean_X = np.concatenate([clean_X_train, clean_X_test])\n",
    "clean_y = np.concatenate([clean_y_train, clean_y_test])\n",
    "clean_X = polynm.transform(clean_X)\n",
    "clean_X = sm.transform(clean_X)\n",
    "\n",
    "clean_sub_data = polynm.transform(clean_sub_data)\n",
    "clean_sub_data = sm.transform(clean_sub_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.03399, std: 0.02060, params: {'n_estimators': 800},\n",
       "  mean: -0.03399, std: 0.02058, params: {'n_estimators': 810},\n",
       "  mean: -0.03400, std: 0.02058, params: {'n_estimators': 820},\n",
       "  mean: -0.03399, std: 0.02058, params: {'n_estimators': 830},\n",
       "  mean: -0.03398, std: 0.02059, params: {'n_estimators': 840},\n",
       "  mean: -0.03397, std: 0.02058, params: {'n_estimators': 850},\n",
       "  mean: -0.03399, std: 0.02059, params: {'n_estimators': 860},\n",
       "  mean: -0.03399, std: 0.02060, params: {'n_estimators': 870},\n",
       "  mean: -0.03400, std: 0.02059, params: {'n_estimators': 880},\n",
       "  mean: -0.03399, std: 0.02059, params: {'n_estimators': 890},\n",
       "  mean: -0.03400, std: 0.02057, params: {'n_estimators': 900}],\n",
       " {'n_estimators': 850},\n",
       " -0.033974821039313596)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# param_test1= {'n_estimators':range(700,801,10)}\n",
    "# gsearch1= GridSearchCV(estimator = RandomForestRegressor(n_estimators=720, max_features='sqrt', random_state=7, n_jobs=12,oob_score=True),\n",
    "#                        param_grid =param_test1, scoring='neg_mean_squared_error',cv=5)\n",
    "# gsearch1.fit(X_train,y_train)\n",
    "# gsearch1.grid_scores_,gsearch1.best_params_, gsearch1.best_score_\n",
    "param_test1= {'n_estimators':range(800,901,10)}\n",
    "gsearch1= GridSearchCV(estimator = LGBMRegressor(n_estimators=800, max_depth=4, random_state=5, n_jobs=12),\n",
    "                       param_grid =param_test1, scoring='neg_mean_squared_error',cv=5)\n",
    "gsearch1.fit(X_train,y_train)\n",
    "gsearch1.grid_scores_,gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.04811, std: 0.02147, params: {'num_leaves': 2},\n",
       "  mean: -0.03958, std: 0.02010, params: {'num_leaves': 3},\n",
       "  mean: -0.03712, std: 0.02000, params: {'num_leaves': 4},\n",
       "  mean: -0.03686, std: 0.02088, params: {'num_leaves': 5},\n",
       "  mean: -0.03590, std: 0.02062, params: {'num_leaves': 6},\n",
       "  mean: -0.03536, std: 0.02031, params: {'num_leaves': 7},\n",
       "  mean: -0.03483, std: 0.02057, params: {'num_leaves': 8},\n",
       "  mean: -0.03508, std: 0.02034, params: {'num_leaves': 9},\n",
       "  mean: -0.03468, std: 0.02065, params: {'num_leaves': 10},\n",
       "  mean: -0.03444, std: 0.02011, params: {'num_leaves': 11},\n",
       "  mean: -0.03414, std: 0.02030, params: {'num_leaves': 12},\n",
       "  mean: -0.03384, std: 0.01991, params: {'num_leaves': 13},\n",
       "  mean: -0.03367, std: 0.02021, params: {'num_leaves': 14},\n",
       "  mean: -0.03413, std: 0.02009, params: {'num_leaves': 15},\n",
       "  mean: -0.03399, std: 0.02060, params: {'num_leaves': 16},\n",
       "  mean: -0.03399, std: 0.02060, params: {'num_leaves': 17},\n",
       "  mean: -0.03399, std: 0.02060, params: {'num_leaves': 18},\n",
       "  mean: -0.03399, std: 0.02060, params: {'num_leaves': 19}],\n",
       " {'num_leaves': 14},\n",
       " -0.03367039687657624)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# param_test2= {'min_samples_split':[2,3]}\n",
    "# gsearch2= GridSearchCV(estimator = RandomForestRegressor(n_estimators=720, max_features='sqrt', random_state=7, n_jobs=12,oob_score=True),\n",
    "#                        param_grid =param_test2, scoring='neg_mean_squared_error',cv=5)\n",
    "# gsearch2.fit(X_train,y_train)\n",
    "# gsearch2.grid_scores_,gsearch2.best_params_, gsearch2.best_score_\n",
    "param_test2= {'num_leaves':range(2,20,1)}\n",
    "gsearch2= GridSearchCV(estimator = LGBMRegressor(n_estimators=800, max_depth=4, random_state=5, n_jobs=12,num_leaves=10),\n",
    "                       param_grid =param_test2, scoring='neg_mean_squared_error',cv=5)\n",
    "gsearch2.fit(X_train,y_train)\n",
    "gsearch2.grid_scores_,gsearch2.best_params_, gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.04811, std: 0.02147, params: {'max_depth': 1},\n",
       "  mean: -0.03780, std: 0.01969, params: {'max_depth': 2},\n",
       "  mean: -0.03488, std: 0.02034, params: {'max_depth': 3},\n",
       "  mean: -0.03367, std: 0.02021, params: {'max_depth': 4},\n",
       "  mean: -0.03358, std: 0.01996, params: {'max_depth': 5},\n",
       "  mean: -0.03416, std: 0.02027, params: {'max_depth': 6},\n",
       "  mean: -0.03400, std: 0.02039, params: {'max_depth': 7},\n",
       "  mean: -0.03439, std: 0.02044, params: {'max_depth': 8},\n",
       "  mean: -0.03427, std: 0.02068, params: {'max_depth': 9},\n",
       "  mean: -0.03445, std: 0.02043, params: {'max_depth': 10},\n",
       "  mean: -0.03425, std: 0.02029, params: {'max_depth': 11},\n",
       "  mean: -0.03423, std: 0.02047, params: {'max_depth': 12}],\n",
       " {'max_depth': 5},\n",
       " -0.03357751517206975)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# param_test3= {'min_samples_split':range(3,4,1), 'min_samples_leaf':range(1,10,1)}\n",
    "# gsearch3= GridSearchCV(estimator = RandomForestRegressor(n_estimators=720, max_features='sqrt', random_state=7, n_jobs=12,oob_score=True),\n",
    "#                        param_grid =param_test3, scoring='neg_mean_squared_error',cv=5)\n",
    "# gsearch3.fit(X_train,y_train)\n",
    "# gsearch3.grid_scores_,gsearch3.best_params_, gsearch3.best_score_\n",
    "param_test3= {'max_depth':range(1,13,1)}\n",
    "gsearch3= GridSearchCV(estimator = LGBMRegressor(n_estimators=800, max_depth=4, random_state=5, n_jobs=12,num_leaves=14),\n",
    "                       param_grid =param_test3, scoring='neg_mean_squared_error',cv=5)\n",
    "gsearch3.fit(X_train,y_train)\n",
    "gsearch3.grid_scores_,gsearch3.best_params_, gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.03395, std: 0.02068, params: {'learning_rate': 0.05},\n",
       "  mean: -0.03358, std: 0.01996, params: {'learning_rate': 0.1},\n",
       "  mean: -0.03542, std: 0.02065, params: {'learning_rate': 0.2},\n",
       "  mean: -0.03624, std: 0.01999, params: {'learning_rate': 0.3},\n",
       "  mean: -0.03834, std: 0.02044, params: {'learning_rate': 0.4},\n",
       "  mean: -0.03996, std: 0.02049, params: {'learning_rate': 0.5},\n",
       "  mean: -0.04097, std: 0.01984, params: {'learning_rate': 0.6},\n",
       "  mean: -0.04551, std: 0.02083, params: {'learning_rate': 0.7}],\n",
       " {'learning_rate': 0.1},\n",
       " -0.03357751517206975)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# param_test4= {'max_features':range(1,5,2)}\n",
    "# gsearch4= GridSearchCV(estimator = RandomForestRegressor(n_estimators=720, max_features='sqrt', random_state=7, n_jobs=12,oob_score=True,min_samples_split=3,max_features=3),\n",
    "#    param_grid = param_test4,scoring='neg_mean_squared_error', cv=5)\n",
    "# gsearch4.fit(X_train,y_train)\n",
    "# gsearch4.grid_scores_,gsearch4.best_params_, gsearch4.best_score_\n",
    "# param_test4= {'boosting':['gbdt','rf','dart','goss']}\n",
    "param_test4= {'learning_rate':[0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]}\n",
    "gsearch4= GridSearchCV(estimator = LGBMRegressor(n_estimators=800, max_depth=5, random_state=5, n_jobs=12,num_leaves=14,boosting='gbrt'),\n",
    "   param_grid = param_test4,scoring='neg_mean_squared_error', cv=5)\n",
    "gsearch4.fit(X_train,y_train)\n",
    "gsearch4.grid_scores_,gsearch4.best_params_, gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.03358, std: 0.01996, params: {}], {}, -0.03357751517206975)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test4= {}\n",
    "gsearch4= GridSearchCV(estimator = LGBMRegressor(tree_learner='data',learning_rate=0.1,n_estimators=800, max_depth=5, random_state=5, n_jobs=12,num_leaves=14,boosting='gbrt'),\n",
    "   param_grid = param_test4,scoring='neg_mean_squared_error', cv=5)\n",
    "gsearch4.fit(X_train,y_train)\n",
    "gsearch4.grid_scores_,gsearch4.best_params_, gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "0.03357751517206975"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tree Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_X_train = np.concatenate([X_train, X_test])\n",
    "all_y_train = np.concatenate([y_train, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8918, 60)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8918,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 - Model: XGBRegressor\n",
      "MSE: [0.0087901  0.02204059 0.01061254 0.08805358 0.01402184]\n",
      "Score: [0.91428109 0.87073068 0.90660418 0.77116567 0.89412341]\n",
      "Average XGB - MSE: 0.028703728819574816  - Score: 0.871381007718373 \n",
      "\n",
      "2 - Model: XGBRegressor\n",
      "MSE: [0.00879629 0.02202212 0.0106034  0.08803991 0.01400969]\n",
      "Score: [0.91425352 0.87077785 0.90664065 0.77117936 0.89416444]\n",
      "Average XGB - MSE: 0.02869428246603136  - Score: 0.8714031635034141 \n",
      "\n",
      "3 - Model: XGBRegressor\n",
      "MSE: [0.00877685 0.02200308 0.01057758 0.08804657 0.01400342]\n",
      "Score: [0.9143402  0.87082651 0.90674381 0.77117269 0.89418561]\n",
      "Average XGB - MSE: 0.028681498574840238  - Score: 0.8714537620446281 \n",
      "\n",
      "4 - Model: GradientBoostingRegressor\n",
      "MSE: [0.01158348 0.01881783 0.01133294 0.0847904  0.01368955]\n",
      "Score: [0.90283134 0.87936974 0.90378627 0.77448061 0.89525325]\n",
      "Average XGB - MSE: 0.028042841739081614  - Score: 0.8711442413773757 \n",
      "\n",
      "5 - Model: GradientBoostingRegressor\n",
      "MSE: [0.0115709  0.01882192 0.0112942  0.08475268 0.0136794 ]\n",
      "Score: [0.90287901 0.87935821 0.90393505 0.77451946 0.89528804]\n",
      "Average XGB - MSE: 0.028023819393120042  - Score: 0.8711959558623767 \n",
      "\n",
      "6 - Model: GradientBoostingRegressor\n",
      "MSE: [0.01157516 0.01882186 0.01130148 0.08477732 0.01368122]\n",
      "Score: [0.90286288 0.87935836 0.90390706 0.77449408 0.89528181]\n",
      "Average XGB - MSE: 0.02803140774699791  - Score: 0.8711808389982046 \n",
      "\n",
      "7 - Model: RandomForestRegressor\n",
      "MSE: [0.01226967 0.02185854 0.01311827 0.08761004 0.01591002]\n",
      "Score: [0.90027756 0.87119675 0.89723516 0.77161093 0.88799305]\n",
      "Average XGB - MSE: 0.030153308790506393  - Score: 0.8656626895206427 \n",
      "\n",
      "8 - Model: RandomForestRegressor\n",
      "MSE: [0.01242509 0.02198718 0.01300311 0.08799085 0.01585023]\n",
      "Score: [0.89971112 0.87086716 0.89764096 0.77122854 0.88818016]\n",
      "Average XGB - MSE: 0.030251290794482445  - Score: 0.8655255865544991 \n",
      "\n",
      "9 - Model: RandomForestRegressor\n",
      "MSE: [0.01219485 0.02162765 0.01277901 0.08776559 0.01589991]\n",
      "Score: [0.90055179 0.87179137 0.89843685 0.77145459 0.88802466]\n",
      "Average XGB - MSE: 0.030053403244699926  - Score: 0.8660518521739731 \n",
      "\n",
      "10 - Model: LGBMRegressor\n",
      "MSE: [0.01253587 0.02279713 0.0112759  0.08504112 0.01471231]\n",
      "Score: [0.89930994 0.86881942 0.90400545 0.77422265 0.89182652]\n",
      "Average XGB - MSE: 0.02927246356598851  - Score: 0.8676367976410448 \n",
      "\n",
      "11 - Model: LGBMRegressor\n",
      "MSE: [0.01322341 0.02310997 0.01136886 0.08546403 0.01527854]\n",
      "Score: [0.89686655 0.86804075 0.90364859 0.77378879 0.88999136]\n",
      "Average XGB - MSE: 0.02968896386507781  - Score: 0.8664672093112552 \n",
      "\n",
      "12 - Model: LGBMRegressor\n",
      "MSE: [0.01363427 0.02324222 0.01137445 0.08388848 0.01544228]\n",
      "Score: [0.89544283 0.8677136  0.90362718 0.77541315 0.88946846]\n",
      "Average XGB - MSE: 0.029516339837112732  - Score: 0.8663330448041281 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Avg MSE</th>\n",
       "      <th>Avg Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.XGBRegressor</td>\n",
       "      <td>0.028704</td>\n",
       "      <td>0.871381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.XGBRegressor</td>\n",
       "      <td>0.028694</td>\n",
       "      <td>0.871403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.XGBRegressor</td>\n",
       "      <td>0.028681</td>\n",
       "      <td>0.871454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.GradientBoostingRegressor</td>\n",
       "      <td>0.028043</td>\n",
       "      <td>0.871144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.GradientBoostingRegressor</td>\n",
       "      <td>0.028024</td>\n",
       "      <td>0.871196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.GradientBoostingRegressor</td>\n",
       "      <td>0.028031</td>\n",
       "      <td>0.871181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.RandomForestRegressor</td>\n",
       "      <td>0.030153</td>\n",
       "      <td>0.865663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.RandomForestRegressor</td>\n",
       "      <td>0.030251</td>\n",
       "      <td>0.865526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.RandomForestRegressor</td>\n",
       "      <td>0.030053</td>\n",
       "      <td>0.866052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.LGBMRegressor</td>\n",
       "      <td>0.029272</td>\n",
       "      <td>0.867637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11.LGBMRegressor</td>\n",
       "      <td>0.029689</td>\n",
       "      <td>0.866467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12.LGBMRegressor</td>\n",
       "      <td>0.029516</td>\n",
       "      <td>0.866333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Model   Avg MSE  Avg Score\n",
       "0                1.XGBRegressor  0.028704   0.871381\n",
       "1                2.XGBRegressor  0.028694   0.871403\n",
       "2                3.XGBRegressor  0.028681   0.871454\n",
       "3   4.GradientBoostingRegressor  0.028043   0.871144\n",
       "4   5.GradientBoostingRegressor  0.028024   0.871196\n",
       "5   6.GradientBoostingRegressor  0.028031   0.871181\n",
       "6       7.RandomForestRegressor  0.030153   0.865663\n",
       "7       8.RandomForestRegressor  0.030251   0.865526\n",
       "8       9.RandomForestRegressor  0.030053   0.866052\n",
       "9              10.LGBMRegressor  0.029272   0.867637\n",
       "10             11.LGBMRegressor  0.029689   0.866467\n",
       "11             12.LGBMRegressor  0.029516   0.866333"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbt1 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=789,eta=0.03,scoring='neg_mean_squared_error',n_estimators=351, max_depth=5, max_features='sqrt', random_state=777, n_jobs=12)\n",
    "xgbt2 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=1000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=361, max_depth=5, max_features='sqrt', random_state=999, n_jobs=12)\n",
    "xgbt3 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=1515,eta=0.03,scoring='neg_mean_squared_error',n_estimators=371, max_depth=5, max_features='sqrt', random_state=367, n_jobs=12)\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=1060, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=1100, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1090, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=740, max_features='sqrt', random_state=7, n_jobs=12)\n",
    "forest2 = RandomForestRegressor(n_estimators=730, max_features='sqrt', random_state=9, n_jobs=12)\n",
    "forest3 = RandomForestRegressor(n_estimators=745, max_features='sqrt', random_state=11, n_jobs=12) \n",
    "# n_estimators 850 num_leaves=14,learning_rate=0.1 max_depth=5  seed\n",
    "lgb1 = LGBMRegressor(n_estimators=840, max_depth=4, random_state=789, n_jobs=12,num_leaves=14,learning_rate=0.08,seed=666) \n",
    "lgb2 = LGBMRegressor(n_estimators=845, max_depth=4, random_state=798, n_jobs=12,num_leaves=13,learning_rate=0.1,seed=777)\n",
    "lgb3 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=777, n_jobs=12,num_leaves=10,learning_rate=0.1,seed=999)\n",
    "\n",
    "# xgbt1 = xgb.XGBRegressor(n_estimators=950, max_depth=3, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "# xgbt2 = xgb.XGBRegressor(n_estimators=1000, max_depth=3, max_features='sqrt', random_state=3, n_jobs=8)\n",
    "# xgbt3 = xgb.XGBRegressor(n_estimators=1100, max_depth=3, max_features='sqrt', random_state=4, n_jobs=8)\n",
    "\n",
    "# gbdt1 = GradientBoostingRegressor(n_estimators=500, max_depth=3, max_features='sqrt', random_state=2)\n",
    "# gbdt2 = GradientBoostingRegressor(n_estimators=400, max_depth=3, max_features='sqrt', random_state=3)\n",
    "# gbdt3 = GradientBoostingRegressor(n_estimators=500, max_depth=4, max_features='log2', random_state=4)\n",
    "\n",
    "# forest1 = RandomForestRegressor(n_estimators=300, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "# forest2 = RandomForestRegressor(n_estimators=300, max_features='log2', random_state=3, n_jobs=8)\n",
    "# forest3 = RandomForestRegressor(n_estimators=600, max_features='sqrt', random_state=4, n_jobs=8) \n",
    "\n",
    "# lgb1 = LGBMRegressor(n_estimators=900, max_depth=5, random_state=2, n_jobs=8) \n",
    "# lgb2 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=3, n_jobs=8)\n",
    "# lgb3 = LGBMRegressor(n_estimators=720, max_depth=4, random_state=4, n_jobs=8)\n",
    "\n",
    "cross_validation_test(\n",
    "    models=[    \n",
    "        xgbt1, xgbt2, xgbt3,\n",
    "        gbdt1, gbdt2, gbdt3,\n",
    "        forest1, forest2, forest3,\n",
    "        lgb1, lgb2, lgb3\n",
    "    ],\n",
    "    train_X_data=[\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train\n",
    "    ],\n",
    "    train_y_data=[\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train\n",
    "    ]\n",
    ")\n",
    "# cross_validation_test(\n",
    "#     models=[    \n",
    "#         xgbt1, xgbt2, xgbt3,\n",
    "#         gbdt1, gbdt2, gbdt3,\n",
    "#         forest1, forest2, forest3,\n",
    "#         lgb1, lgb2, lgb3\n",
    "#     ],\n",
    "#     train_X_data=[\n",
    "#         all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "#         all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "#         all_X_train, all_X_train, all_X_train, all_X_train\n",
    "#     ],\n",
    "#     train_y_data=[\n",
    "#         all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "#         all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "#         all_y_train, all_y_train, all_y_train, all_y_train\n",
    "#     ]\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbt1 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=3000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=351, max_depth=5, max_features='sqrt', random_state=2, n_jobs=12)\n",
    "xgbt2 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=3000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=361, max_depth=5, max_features='sqrt', random_state=3, n_jobs=12)\n",
    "xgbt3 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=3000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=371, max_depth=5, max_features='sqrt', random_state=4, n_jobs=12)\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=1010, max_depth=5, max_features='log2', random_state=790,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=1000, max_depth=5, max_features='log2', random_state=791,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1020, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=720, max_features='sqrt', random_state=7, n_jobs=12)\n",
    "forest2 = RandomForestRegressor(n_estimators=750, max_features='log2', random_state=9, n_jobs=12)\n",
    "forest3 = RandomForestRegressor(n_estimators=800, max_features='sqrt', random_state=11, n_jobs=12) \n",
    "\n",
    "lgb1 = LGBMRegressor(n_estimators=800, max_depth=4, random_state=5, n_jobs=12) \n",
    "lgb2 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=7, n_jobs=12)\n",
    "lgb3 = LGBMRegressor(n_estimators=900, max_depth=4, random_state=9, n_jobs=12)\n",
    "\n",
    "Model\tAvg MSE\tAvg Score\n",
    "0\t1.XGBRegressor\t0.028357\t0.871174\n",
    "1\t2.XGBRegressor\t0.028332\t0.871251\n",
    "2\t3.XGBRegressor\t0.028333\t0.871279\n",
    "3\t4.GradientBoostingRegressor\t0.027985\t0.871623\n",
    "4\t5.GradientBoostingRegressor\t0.027273\t0.872185\n",
    "5\t6.GradientBoostingRegressor\t0.026847\t0.874907\n",
    "6\t7.RandomForestRegressor\t0.030499\t0.865014\n",
    "7\t8.RandomForestRegressor\t0.030333\t0.864953\n",
    "8\t9.RandomForestRegressor\t0.030495\t0.864881\n",
    "9\t10.LGBMRegressor\t0.029263\t0.868385\n",
    "10\t11.LGBMRegressor\t0.029263\t0.868362\n",
    "11\t12.LGBMRegressor\t0.029246\t0.868430"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "xgbt1 = xgb.XGBRegressor(n_estimators=950, max_depth=3, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "xgbt2 = xgb.XGBRegressor(n_estimators=1000, max_depth=3, max_features='sqrt', random_state=3, n_jobs=8)\n",
    "xgbt3 = xgb.XGBRegressor(n_estimators=1100, max_depth=3, max_features='sqrt', random_state=4, n_jobs=8)\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=800, max_depth=4, max_features='log2', random_state=2,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=900, max_depth=4, max_features='log2', random_state=3,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1000, max_depth=5, max_features='log2', random_state=4,learning_rate=0.08)\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=300, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "forest2 = RandomForestRegressor(n_estimators=400, max_features='log2', random_state=3, n_jobs=8)\n",
    "forest3 = RandomForestRegressor(n_estimators=600, max_features='sqrt', random_state=4, n_jobs=8) \n",
    "\n",
    "lgb1 = LGBMRegressor(n_estimators=900, max_depth=5, random_state=2, n_jobs=8) \n",
    "lgb2 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=3, n_jobs=8)\n",
    "lgb3 = LGBMRegressor(n_estimators=720, max_depth=4, random_state=4, n_jobs=8)\n",
    "\n",
    "Model\tAvg MSE\tAvg Score\n",
    "0\t1.XGBRegressor\t0.029047\t0.869022\n",
    "1\t2.XGBRegressor\t0.029007\t0.869151\n",
    "2\t3.XGBRegressor\t0.028923\t0.869379\n",
    "3\t4.GradientBoostingRegressor\t0.028264\t0.870069\n",
    "4\t5.GradientBoostingRegressor\t0.028909\t0.869134\n",
    "5\t6.GradientBoostingRegressor\t0.027992\t0.872356\n",
    "6\t7.RandomForestRegressor\t0.030729\t0.864344\n",
    "7\t8.RandomForestRegressor\t0.030402\t0.865025\n",
    "8\t9.RandomForestRegressor\t0.030547\t0.864977\n",
    "9\t10.LGBMRegressor\t0.028749\t0.868748\n",
    "10\t11.LGBMRegressor\t0.029332\t0.867618\n",
    "11\t12.LGBMRegressor\t0.029250\t0.867823\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## randomForest 还可以调参再强化一下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Model\tAvg MSE\tAvg Score\n",
    "0\t1.XGBRegressor\t0.031286\t0.862379\n",
    "1\t2.XGBRegressor\t0.031232\t0.862519\n",
    "2\t3.XGBRegressor\t0.031166\t0.862695\n",
    "3\t4.GradientBoostingRegressor\t0.030385\t0.863508\n",
    "4\t5.GradientBoostingRegressor\t0.031893\t0.859239\n",
    "5\t6.GradientBoostingRegressor\t0.028836\t0.868424\n",
    "6\t7.RandomForestRegressor\t0.030334\t0.865056\n",
    "7\t8.RandomForestRegressor\t0.030304\t0.865000\n",
    "8\t9.RandomForestRegressor\t0.030405\t0.864965\n",
    "9\t10.LGBMRegressor\t0.029516\t0.866988\n",
    "10\t11.LGBMRegressor\t0.029343\t0.867077\n",
    "11\t12.LGBMRegressor\t0.029338\t0.867135\n",
    "Score: 0.8678230918561365 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "# knn1 = KNeighborsRegressor(n_neighbors=7, p=1)\n",
    "# knn2 = KNeighborsRegressor(n_neighbors=8, p=2)\n",
    "# knn3 = KNeighborsRegressor(n_neighbors=6, p=1)\n",
    "# cross_validation_test(\n",
    "#     models=[knn1, knn2, knn3],\n",
    "#     train_X_data=[\n",
    "#         all_X_train, all_X_train, all_X_train, \n",
    "#     ],\n",
    "#     train_y_data=[\n",
    "#         all_y_train, all_y_train, all_y_train,\n",
    "#     ]\n",
    "# )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "regrs = [\n",
    "    xgbt1, gbdt1, forest1, lgb1,\n",
    "    xgbt2, gbdt2, forest2, lgb2,\n",
    "    xgbt3, gbdt3, forest3, lgb3\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stacker(object):\n",
    "    def __init__(self, n_splits, stacker, base_models):\n",
    "        self.n_splits = n_splits\n",
    "        self.stacker = stacker\n",
    "        self.base_models = base_models\n",
    "    \n",
    "    # X: 原始训练集, y: 原始训练集真实值, predict_data: 原始待预测数据\n",
    "    def fit_predict(self, X, y, predict_data):\n",
    "        X = np.array(X)\n",
    "        y = np.array(y)\n",
    "        T = np.array(predict_data)\n",
    "\n",
    "        folds = list(KFold(n_splits=self.n_splits, shuffle=False, random_state=2018).split(X, y))\n",
    "        \n",
    "        # 以基学习器预测结果为特征的 stacker的训练数据 与 stacker预测数据\n",
    "        S_train = np.zeros((X.shape[0], len(self.base_models)))\n",
    "        S_predict = np.zeros((T.shape[0], len(self.base_models)))\n",
    "        \n",
    "        for i, regr in enumerate(self.base_models):\n",
    "            print(i + 1, 'Base model:', str(regr).split('(')[0])\n",
    "            S_predict_i = np.zeros((T.shape[0], self.n_splits))\n",
    "            \n",
    "            for j, (train_idx, test_idx) in enumerate(folds):\n",
    "                # 将X分为训练集与测试集\n",
    "                X_train, y_train, X_test, y_test = X[train_idx], y[train_idx], X[test_idx], y[test_idx]\n",
    "                print ('Fit fold', (j+1), '...')\n",
    "                regr.fit(X_train, y_train)\n",
    "                y_pred = regr.predict(X_test)                \n",
    "                S_train[test_idx, i] = y_pred\n",
    "                S_predict_i[:, j] = regr.predict(T)\n",
    "            \n",
    "            S_predict[:, i] = S_predict_i.mean(axis=1)\n",
    "\n",
    "        nmse_score = cross_val_score(self.stacker, S_train, y, cv=5, scoring='neg_mean_squared_error')\n",
    "        print('CV MSE:', -nmse_score)\n",
    "        print('Stacker AVG MSE:', -nmse_score.mean(), 'Stacker AVG Score:', np.mean(np.divide(1, 1 + np.sqrt(-nmse_score))))\n",
    "\n",
    "        self.stacker.fit(S_train, y)\n",
    "        res = self.stacker.predict(S_predict)\n",
    "        return res, S_train, S_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "2 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "3 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "4 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "5 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "6 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "7 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "8 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "9 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "10 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "11 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "12 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "CV MSE: [0.00887032 0.01872151 0.00867298 0.08544655 0.01221285]\n",
      "Stacker AVG MSE: 0.026784839853493135 Stacker AVG Score: 0.8765327586171049\n"
     ]
    }
   ],
   "source": [
    "# stacking_mode1 = Ridge(alpha=0.008, copy_X=True, fit_intercept=False, solver='auto', random_state=2)\n",
    "stacking_model = SVR(C=100, gamma=0.01, epsilon=0.01)\n",
    "stacker = Stacker(5, stacking_model, regrs)\n",
    "pred_stack, S_train_data, S_predict_data = stacker.fit_predict(all_X_train, all_y_train, sub_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV MSE: [0.00834157 0.01945687 0.0095695  0.08376848 0.01302208]\n",
    "Stacker AVG MSE: 0.02683170221309707 Stacker AVG Score: 0.8755806306487525"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "2 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "3 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "4 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "5 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "6 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "7 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "8 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "9 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "10 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "11 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "12 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "CV MSE: [0.00911334 0.01872236 0.05803914 0.01062331 0.00985086]\n",
      "Stacker AVG MSE: 0.02126980198246859 Stacker AVG Score: 0.8829247893247889\n"
     ]
    }
   ],
   "source": [
    "stacking_model2 = SVR(C=100, gamma=0.01, epsilon=0.01)\n",
    "stacker2 = Stacker(5, stacking_model2, regrs)\n",
    "pred_clean_stack, S_clean_train_data, S_clean_predict_data = stacker2.fit_predict(clean_X, clean_y, clean_sub_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV MSE: [0.01659384 0.00626714 0.01261921 0.05345451 0.01616147]\n",
    "Stacker AVG MSE: 0.021019233812974467 Stacker AVG Score: 0.8821919685074249"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV MSE: [0.00902831 0.01733493 0.05746076 0.01078939 0.00961003]\n",
    "Stacker AVG MSE: 0.020844682710295333 Stacker AVG Score: 0.8840294650706134"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_result['score'] = pred_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "index = df_result[df_result['ID'].isin(special_missing_ID)].index\n",
    "df_result.loc[index, 'score'] = 0.379993053"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_index = df_result[df_result['ID'].isin(cleaned_sub_data_ID)].index\n",
    "df_result.loc[c_index, 'score'] = pred_clean_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv('8_9_time_21.32.version4.0.csv', index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
